# -*- coding: utf-8 -*-
'''
Created on 30-03-2013

@author: LONG HOANG GIANG
'''
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from urlparse import urljoin
import pyLib
import datetime
import re
import json
import workerpool


class VFicLand():
    
    url = ''
    path = ''
    filesPath = ''
    
    def __init__(self, url):
        pass
    
    def getDetail(self, item):
        print '> getDetail {0}'.format(item['url'])
        tree = pyLib.loadWeb(item['url']).build_tree()
        contentNode = tree.xpath("//div[@id='content']/div[contains(@id, 'post-')]/div[@class='entry-content clearfix']")[0]
        html = pyLib.cleanHTML(pyLib.Etree.tostring(contentNode), ['b', 'strong', 'hr'])
        print html
        html = re.sub(r"\n", "<br />", html)
        html = """<b>{0}</b><br /><hr /><br />{1}""".format(item['title'], html)
        pyLib.gzip(self.filesPath + item['id'], json.dumps(html))
    
    def process(self, url):
        
        if not url.endswith('/'): url += '/'
        self.url = url
        folderName = pyLib.extractText(r"/([^/]+)/$", url, 1)
        self.path = "/longhoanggiang/truyentext/{0}/".format(folderName)
        self.filesPath = self.path + 'files/'
        if not os.path.isdir(self.filesPath): os.makedirs(self.filesPath, 0777)
        
        tree = pyLib.loadWeb(self.url, cached=True, referer='http://vficland.com/').build_tree()
        data = []
        for item in tree.xpath("//ul[@class='lcp_catlist']/li/a"):
            title = VFicLand.getTitle(pyLib.stringify(item))
            link = item.get('href', '')
            if link == '': continue
            link = urljoin(self.url, link)
            print title, link
            fileName = pyLib.crc32unsigned(link)
            data.append({'title': title, 'id': fileName, 'url': link})
        
        pool = workerpool.WorkerPool(size=2)
        pool.map(self.getDetail, data)
        pool.shutdown()
        pool.wait()
        
        print '------------------------------- finished crawler detail'
        
        for item in data: 
            del item['url']
            
        data.reverse()
        pyLib.gzip(self.path + 'data', json.dumps(data))
        
        return
    
            
    @staticmethod
    def getTitle(strInput):
#        return pyLib.extractText(r"\[(.+)\]", strInput, 1)
        return strInput

if __name__ == '__main__':
    
    print '> Start crawler vficland'
    
    listStories = [
#                   'http://vficland.com/truyen-phuong-dong/tay-om-con-tay-om-vo/',
#                   'http://vficland.com/truyen-phuong-dong/doc-duoc-phong-ban-ve/',
#                   'http://vficland.com/truyen-phuong-tay/vong-xoay-dam-me/',
#                   'http://vficland.com/truyen-phuong-dong/yeu-gia-tinh-that/',
#                   'http://vficland.com/bai-viet-va-truyen-sang-tac/tinh-yeu-co-noi-tiep-kiep-sau/',
#                   'http://vficland.com/truyen-phuong-dong/tong-tai-ba-dao/',
#                   'http://vficland.com/truyen-phuong-dong/nguoi-tinh-giau-mat/',
#                   'http://vficland.com/truyen-phuong-dong/cuc-cung-kieu-ngao-pk-tong-tai-papa/',
#                   'http://vficland.com/truyen-phuong-dong/khong-yeu-thi-the-nao/',
#                   'http://vficland.com/truyen-phuong-dong/nhung-cau-chuyen-ke-voi/'
#                    'http://vficland.com/truyen-phuong-dong/boss-qua-gian-xao/',
#                    'http://vficland.com/truyen-phuong-dong/36-chieu-ly-hon/',
#                    'http://vficland.com/truyen-phuong-dong/ma-nu-tinh-thu/',
#                    'http://vficland.com/truyen-phuong-dong/me-hoac-vuong-tu-dang-yeu/',
#                    'http://vficland.com/truyen-phuong-dong/neu-nhu-anh-yeu-em/',
#                    'http://vficland.com/truyen-phuong-dong/the-nao-mot-loai-yeu-khong-dau/',
#                    'http://vficland.com/truyen-phuong-dong/thu-yeu-mot-lan/'
#                    'http://vficland.com/truyen-phuong-dong/cap-doi-xui-xeo/',
#                    'http://vficland.com/truyen-phuong-dong/ai-hieu-duoc-long-em/',
#                    'http://vficland.com/truyen-phuong-dong/me-vi-thanh-nien/',
#                    'http://vficland.com/truyen-phuong-dong/neu-nhu-anh-yeu-em/',
#                    'http://vficland.com/truyen-phuong-dong/no-em-mot-hanh-phuc/',
                    'http://vficland.com/truyen-phuong-dong/tan-sung/',
                   ]
    
    for url in listStories:
        a = VFicLand()
        a.process(url)
        
#    a = VFicLand('http://vficland.com/truyen-phuong-dong/boss-qua-gian-xao/')
#    a.process()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)  
    
